package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.KaiDiModuleTask;
import com.spider.silence.task.KaiDiTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;


import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by Administrator on 2017/1/9 0009.
 */
public class KaiDiDownLoader extends AbstractCommonDownLoader  implements Cloneable{
    private Logger logger = Logger.getLogger(KaiDiDownLoader.class);
    private Map<String,File> files=null;

    public Map<String,File> getFiles(){

        return files;
    }

    private String url = "http://kdnet.net/";

    /*private String[] modules={
            "http://club.kdnet.net/list.asp?boardid=1",//
            "http://club.kdnet.net/list.asp?boardid=89",
            "http://club.kdnet.net/list.asp?boardid=52",
            "http://club.kdnet.net/list.asp?boardid=3",

    };*/

    public List<String> getUrl(){
        List<String> urlList = new ArrayList<String>();
        //String sub_forum_url = "http://kdnet.net/clubforum1.asp";
        String sub_forum_url2 = "http://club.kdnet.net/index.asp";
        Document document = null;
        long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.URLWaitTime"));
        long remianing = Long.valueOf(PropertiesUtil.getVal("KaiDi.URLWaitTime"));
        boolean flag = true;
      /*  while(flag&&remianing>=0) {
            try {
                remianing = future - System.currentTimeMillis();
                *//*Page pageHtml= new Page();
                pageHtml = getHtml(sub_forum_url,true);
                if(pageHtml == null ||pageHtml.getStatusCode()!=200){
                    continue;
                }*//*
                document = get(sub_forum_url, ProxyPool.proxy());

                Elements sub_forum = document.select(".clubforum-sub");
                for (int i = 0; i < sub_forum.size(); i++) {
                    Elements sub_forums = sub_forum.get(i).select("a");
                    for (int j = 0; j < sub_forums.size(); j++) {
                        String sub_forum_url1 = sub_forums.get(j).attr("href");
                        urlList.add(sub_forum_url1);
                    }
                }
                flag = false;
                //System.out.println(urlList);
            } catch (Exception e) {
                logger.error(e);
//                e.printStackTrace();
                break;
            }
        }*/

        if(urlList==null||urlList.size()<=0){
            flag = true;
            future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.URLWaitTime"));
            while (flag&&remianing>=0) {
                try {
                    remianing = future - System.currentTimeMillis();
                    document = get(sub_forum_url2,  true);

                    //Elements sub_forum = document.select(".index-table").select("table[id^='gb-forumlist']");
                    Elements sub_forum = document.select("div[class=\"bbsindex\"] > div[class=\"bbsCont\"] > div[class=\"bbsLeft\"] > div[class=\"hotplate\"] > ul > li");
                    for (int i = 0; i < sub_forum.size(); i++) {
                        if(!"更多".equals(sub_forum.get(i).text())){
                            String forum_url =  sub_forum.get(i).select("a").get(0).attr("href");
                            urlList.add("http:" + forum_url);
                        }
                        /*String id = sub_forum.get(i).attr("id");
                        String regex = "gb-forumlist[0-9]{1,2}";
                        if (id.matches(regex)) {
                            Elements trs = sub_forum.get(i).select("tr");

                            String sub_name = trs.get(0).text().trim();
                            if (!"站务".equals(sub_name) && !"编辑事务".equals(sub_name)) {
                                for (int j = 1; j < trs.size(); j++) {
                                    String forum_url = trs.get(j).select("td").get(1).select("a").get(0).attr("href");
                                    urlList.add(forum_url);
                                }
                            }
                        }*/
                    }
                    flag = false;
                } catch (Exception e) {
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                }
            }
        }
/*
        long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.URLWaitTime"));
        long remianing = Long.valueOf(PropertiesUtil.getVal("KaiDi.URLWaitTime"));
        boolean flag = true;
        while (flag&&remianing>=0) {
            try {
                remianing = future - System.currentTimeMillis();
                document = get(sub_forum_url, true);
                Elements sub_forum = document.select(".index-table").select("table[id^='gb-forumlist']");
                for (int i = 0; i < sub_forum.size(); i++) {
                    String id = sub_forum.get(i).attr("id");
                    String regex = "gb-forumlist[0-9]{1,2}";
                    if (id.matches(regex)) {
                        Elements trs = sub_forum.get(i).select("tr");

                        String sub_name = trs.get(0).text().trim();
                        if (!"站务".equals(sub_name) && !"编辑事务".equals(sub_name)) {
                            for (int j = 1; j < trs.size(); j++) {
                                String forum_url = trs.get(j).select("td").get(1).select("a").get(0).attr("href");
                                urlList.add(forum_url);
                            }
                        }
                    }
                }
                flag = false;
            } catch (Exception e) {
                e.printStackTrace();
            }
        }*/

        return urlList;
    }

    public void generateTask(List<String> urlList){
        for(int i = 0;i<urlList.size();i++){
           /*ExecutorService pool = Executors.newFixedThreadPool(5);
            try {
                long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.GetForumUrlTime"));
                long remianing = Long.valueOf(PropertiesUtil.getVal("KaiDi.GetForumUrlTime"));
                boolean flag = true;
                List<Elements> elementsList = new ArrayList<Elements>();
                //List<FutureTask<Elements>> futureTasks = new ArrayList<java.util.concurrent.FutureTask<Elements>>();
                while (flag && remianing >= 0) {
                    try {
                        remianing = future - System.currentTimeMillis();
                        Document document = get(urlList.get(i), ProxyPool.proxy());

                        String sub_url = urlList.get(i);

                        Elements threadList = document.select(".wrapper").select(".list-table").select("tr[name^='showreply_']");
                        Elements pageElements = document.select(".wrapper").select(".operating").select(".pages").select(".pagesmodule");

                        *//*for (int j = 0; j < threadList.size(); j++) {
                            String forumUrl = threadList.get(j).select(".subject").select(".f14px").select("a").get(0).attr("href");
                            KaiDiTask task = new KaiDiTask(URLDecoder.decode(forumUrl, "UTF-8"));
                            CommonTaskManager.add("KaiDiForum", task);
                            System.out.println(forumUrl);
                        }*//*

                        if(threadList.size()>0&&threadList!=null){
                            elementsList.add(threadList);
                        }

                        flag = false;

                        String pageInfo = pageElements.text();
                       // System.out.println(pageInfo);

                        String regex = "/[0-9]{0,}";
                        Pattern pattern = Pattern.compile(regex);
                        Matcher matcher = pattern.matcher(pageInfo);
                        String all = "";
                        int total_page = 0;
                        if(matcher.find()) {
                            all = matcher.group();
                            all = all.replaceAll("/","");
                            total_page =  Integer.valueOf(all);
                            System.out.println("总页数"+all);
                        }


                        ExecutorService pool2 = Executors.newFixedThreadPool(50);
                        ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, total_page, 50);
                        if(total_page>=2) {
                            for (int k = 0; k <splitPages.size(); k++) {
                                SubForumUrl crawler = new SubForumUrl(sub_url,splitPages.get(k));
                                pool2.execute(crawler);//提交任务
                            }
                        }

                        //String all = matcher.replaceAll("");

                        *//*for(int k=0;k<pageElements.size();k++){
                            String text = pageElements.get(k).text();
                            if("下一页".equals(text)){
                                String new_url = pageElements.get(k).attr("href");
                                generateNextTask(new_url);
                            }
                        }*//*

                    } catch (Exception e) {
                        logger.error(e.getMessage(),e);
                        continue;
                    }
                }

                *//*for(FutureTask<Elements> futuretask:futureTasks){
                    Elements threadList1 = futuretask.get();
                    if(threadList1!=null&&threadList1.size()>0){
                        elementsList.add(threadList1);
                    }
                }*//*

                ArrayList<ArrayList<Integer>> taskSplit = CommonUtils.splitPages(1, elementsList.size(), 5);
                for(int j = 0;j<taskSplit.size();j++){
                    ModuleCrawler crawler = new ModuleCrawler(elementsList, taskSplit.get(j));
                    pool.execute(crawler);//提交任务
                }
            }catch (Exception e){
                logger.error(e.getMessage(),e);
                continue;
            }*/

            try {
                KaiDiModuleTask task = new KaiDiModuleTask(urlList.get(i));
                CommonTaskManager.add("KaiDiModule", task);
            }catch (Exception e){
                logger.error(e.getMessage(),e);
            }

        }
    }


    class SubForumUrl extends AbstractCommonDownLoader implements Runnable{
        private Logger logger = Logger.getLogger(SubForumUrl.class);
        private String forum_url;
        private ArrayList<Integer> pages;


        public SubForumUrl(String forum_url, ArrayList<Integer> pages) {
            this.forum_url = forum_url;
            this.pages = pages;
        }


        @Override
        public void run() {

            for (int k = 0; k < pages.size(); k++) {
                String new_url = forum_url + "&page=" + pages.get(k);
                long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.GetForumUrlTime"));
                boolean flag1 = true;
                long remianing1 = Long.valueOf(PropertiesUtil.getVal("KaiDi.GetForumUrlTime"));
                Elements threadList1 = null;
                Document document1 = null;
                while (flag1 && remianing1 >= 0) {
                    try {
                        remianing1 = future1 - System.currentTimeMillis();
                        Page page = new Page();
                        page = getHtml(new_url, true);
                        if (page != null) {
                            if (page.getStatusCode() != 200 || page.getRawText() == null) {
                                System.out.println(page.getStatusCode());
                                continue;
                            } else {
                                document1 = page.getHtml().getDocument();
                            }
                        } else {
                            continue;
                        }
                     /*document1 = get(forum_url, true);*/
                        threadList1 = document1.select(".wrapper").select(".list-table").select("tr[name^='showreply_']");
                        for (int i = 0; i < threadList1.size(); i++) {
                            String forumUrl = threadList1.get(i).select(".subject").select(".f14px").select("a").get(0).attr("href");
                            KaiDiTask task = new KaiDiTask(URLDecoder.decode(forumUrl, "UTF-8"));
                            CommonTaskManager.add("KaiDiForum", task);
                            System.out.println(forumUrl);
                        }
                        flag1 = false;
                    } catch (Exception e) {
                        logger.error("解析模块中其他页内容错误", e);
                        continue;
                    }
                }

            }
        }
    }


    class ModuleCrawler extends AbstractCommonDownLoader implements Runnable {
        private Logger logger = Logger.getLogger(ModuleCrawler.class);
        private List<Elements> threadList;
        private ArrayList<Integer> taskIndex;

        public ModuleCrawler(List<Elements> elements, ArrayList<Integer> taskIndex) {
            this.threadList = elements;
            this.taskIndex = taskIndex;
        }

        @Override
        public void run() {
            for (int k = 0; k < taskIndex.size(); k++) {
                //int index = taskIndex.get(k) - 1;
                //String url = elements.get(index).attr("href");
                try {
                    //Document document = get(url);
                    //Elements forumElements = document.select("tbody[id^='normalthread_'] a.xst");
                    for (int i = 0; i < threadList.size(); i++) {
                        Elements threads = threadList.get(i);
                        for(int j = 0;j<threads.size();j++) {
                            String forumUrl = threads.get(j).select(".subject").select(".f14px").select("a").get(0).attr("href");
                            KaiDiTask task = new KaiDiTask(URLDecoder.decode(forumUrl, "UTF-8"));
                            CommonTaskManager.add("KaiDiForum", task);
                            System.out.println(forumUrl);
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    e.printStackTrace();
                }
            }
        }
    }



   /* public void generateNextTask(String url){
        try{
            //remianing = future - System.currentTimeMillis();
            Document document = get(url, true);

            Elements pageElements = document.select(".wrapper").select(".operating").select(".pages").select(".pagesmodule").select("a");

            Elements threadList = document.select(".wrapper").select(".list-table").select("tr[name^='showreply_']");
            for (int j = 0; j < threadList.size(); j++) {
                String forumUrl = threadList.get(j).select(".subject").select(".f14px").select("a").get(0).attr("href");
                KaiDiTask task = new KaiDiTask(URLDecoder.decode(forumUrl, "UTF-8"));
                CommonTaskManager.add("KaiDiForum", task);
                System.out.println(forumUrl);
            }

                    *//*flag = false;*//*

            for(int k=0;k<pageElements.size();k++){
                String text = pageElements.get(k).text();
                if("下一页".equals(text)){
                    String new_url = pageElements.get(k).attr("href");
                    generateNextTask(new_url);
                }
            }

        }catch (Exception e){
            logger.equals(e.getMessage());
        }
    }*/

    public JSONObject crawlItem(String content_url){
        files=new ConcurrentHashMap<>();
        JSONObject item=new JSONObject();
        JSONObject data = new JSONObject();
        //content_url = "http://club.kdnet.net/dispbbs.asp?id=12066727&boardid=1";
        int page=1;
        try {

            //boolean flag = true;
            Document document = null;
            //long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("KaiDi.WaitTime"));
            // long remianing = Long.valueOf(PropertiesUtil.getVal("KaiDi.WaitTime"));
            //while (flag && remianing >= 0) {
            try {
                /*Page htmlPage = new Page();
                htmlPage = getHtml(content_url, ProxyPool.proxy());*/

                Page pageContent = null;
                try {
                    pageContent = getHtml(content_url);

                    if (pageContent == null && pageContent.getStatusCode() != 200) {
                        pageContent = getHtml(content_url, true);
                    }
                }catch (Exception e){
                    pageContent = getHtml(content_url, true);
                    e.printStackTrace();
                }

                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                    try {
                        document = pageContent.getHtml().getDocument();
                        Elements contentsInfo = document.select(".posted-box-add");

                        String publish_user = contentsInfo.select(".posted-box").select(".posted-info").select("#userinfo_1").select(".name").select("a").get(0).text();
                        String user_url = contentsInfo.select(".posted-box").select(".posted-info").select("#userinfo_1").select(".name").select("a").get(0).attr("href");
                        String content_id = contentsInfo.select(".posted-box").select("a").get(0).attr("name");

                        String publish_user_photo = "";

                        String publish_user_id = "";
                        if (user_url != null && !"".equals(user_url)) {
                            publish_user_id = user_url.split("\\?")[1].split("=")[1];
                            publish_user_photo = "http://qc-cache.kdnet.net//upload/userface/" + publish_user_id + ".jpg?imageMogr2/crop/190x190";
                               /*Document document1 = null;
                                try {
                                    document1 = get(user_url);
                                    publish_user_photo = document1.select("#content").select(".n_gb_c").select(".userinfo").select(".userpic").select("a").get(0).select("#userface_img_index").attr("src");
                                } catch (Exception e) {
                                    logger.error(e.getMessage(),e);
                                }*/
                        }

                        //Map<String, Object> map = getUrlParams(content_url);

                        String id = content_url.split("\\?")[1].split("&")[0].split("=")[1];
                        boolean result=id.matches("[0-9]+");
                        if (result != true) {
                            //if ("share".equals(id)) {
                            id = content_url.split("\\?")[1].split("&")[1].split("=")[1];
                        }
                        int read_count = 0;
                        int reply_count = 0;
                        //System.out.println(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").size());
                        if(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").size()==5) {
                            read_count = Integer.valueOf(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").get(0).select("span").get(0).text());
                            reply_count = Integer.valueOf(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select(".forward-wblog").select("span").get(0).text());
                        }else{
                            read_count = Integer.valueOf(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").get(0).select("span").get(0).text());
                            reply_count = Integer.valueOf(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").get(2).select("span").get(0).text());
                            //System.out.println(contentsInfo.select(".postspecific").select(".posts-stat-box").select(".posts-stat-c").select("div").get(0).text());
                        }
                        String title = document.title();

                        page = (int) Math.ceil(((double) reply_count + 1) / 15d);//页数

                        data.put("id", id);
                        data.put("publish_user", publish_user);
                        data.put("publish_user_id", publish_user_id);
                        data.put("content_id", content_id);
                        data.put("publish_user_photo", publish_user_photo);
                        data.put("read_count", read_count);
                        data.put("reply_count", reply_count);
                        data.put("title", title);
                        data.put("url", content_url);

                        String publish_time = contentsInfo.select(".postspecific").select(".posts-posted").text().split(" ")[2] + " " + contentsInfo.select(".postspecific").select(".posts-posted").text().split(" ")[3];
                        publish_time = publish_time.replaceAll("/", "-");
                        String content = contentsInfo.select(".postspecific").select(".posts-cont").html();
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=contentsInfo.select(".postspecific").select(".posts-cont").select("img");
                        for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                            Element imgElement=imgElements.get(imgIndex);
                            String img_url=imgElement.absUrl("src");
                            img_urls.add(img_url);
                        }
                        String spider_time = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);
                        /*System.out.println(contentsInfo.select(".posts-control").select("span[id^='ding_']"));

                         if (contentsInfo.select(".posts-control").select("span[id^=ding_]").select("a").get(0).select("span").size() > 1) {
                            String like_count = contentsInfo.select(".posts-control").select("span[id^=ding_]").select("a").get(0).select("span").get(1).text();
                        }
                         if (contentsInfo.select(".posts-control").select("span[id^=ding_]").select("a").get(1).select("span").size() > 1) {
                            String dislike_count = contentsInfo.select(".posts-control").select("span[id^=ding_]").select("a").get(1).select("span").get(1).text();
                        }*/
                        data.put("publish_time", publish_time);
                        data.put("content", FileUtils.filterImg(content));
                        data.put("img_urls",img_urls);
                        data.put("spider_time", spider_time);

                        long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                        String filePrefix="KaiDi";//文件前缀
                        String fileName=filePrefix+"_"+tsp+"_"+publish_user_id+"_"+id;
                        String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                        String speechesId="KaiDi_speeches_"+publishDate+"_"+fileName;
                        File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                        files.put(speechesId+"_pageNo_1",file);
                        //帖子首页的文件

                        JSONArray reply_nodes = new JSONArray();

                        Elements replyInfos = document.select(".reply-box");
                        //查询本页的回复信息
                        for (int i = 0; i < replyInfos.size(); i++) {
                            try {
                                JSONObject reply_object = new JSONObject();
                                String reply_id = replyInfos.get(i).attr("id");
                                String reply_publish_user_url = replyInfos.get(i).select(".posted-box").select(".posted-info").select(".name").select("span").get(0).select("a").get(0).attr("href");
                                String reply_publish_user_id = "";
                                String reply_publish_user_photo = "";
                                if (reply_publish_user_url != null && !"".equals(reply_publish_user_url)) {
                                    reply_publish_user_id = reply_publish_user_url.split("\\?")[1].split("=")[1];
                                    reply_publish_user_photo = "http://qc-cache.kdnet.net//upload/userface/" + reply_publish_user_id + ".jpg?imageMogr2/crop/190x190";
                                      /*  Document document1 = null;
                                        try {
                                            document1 = get(reply_publish_user_url);
                                            reply_publish_user_photo = document1.select("#content").select(".n_gb_c").select(".userinfo").select(".userpic").select("a").get(0).select("#userface_img_index").attr("src");
                                        } catch (Exception e) {
                                             logger.error(e.getMessage());
                                        }*/
                                }
                                String reply_publish_user = replyInfos.get(i).select(".posted-box").select(".posted-info").select(".name").select("span").get(0).select("a").get(0).text();
                                String reply_time = replyInfos.get(i).select(".posted-box").select(".posted-info").text();

                                reply_time = reply_time.split("\\|")[3].split(" ")[1] + " " + reply_time.split("\\|")[3].split(" ")[2];
                                reply_time = reply_time.replaceAll("/", "-");
                                String reply_spider_time = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);
                                String reply_content = replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").html();
                                JSONArray reply_img_urls=new JSONArray();
                                Elements replyImgElements=replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").select("img");
                                for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                    Element imgElement=replyImgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    reply_img_urls.add(img_url);
                                }
                                String reply_url = content_url + "#" + reply_id;

                                String parent_id = "";
                                String ancestor_id = "";
                                Elements reply_reply = replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").select(".quote-cont-box");
                                if (reply_reply.size() > 0 && reply_reply != null) {
                                    String parent_content_url = reply_reply.select(".quote-cont2").select(".quote-title").select("a").get(0).attr("href");
                                    parent_id = getUrlParams(parent_content_url).get("replyid").toString();
                                    ancestor_id = id;
                                } else {
                                    parent_id = id;
                                    ancestor_id = id;
                                }
                                reply_object.put("id", reply_id);
                                reply_object.put("publish_user", reply_publish_user);
                                reply_object.put("publish_user_id", reply_publish_user_id);
                                reply_object.put("publish_user_photo", reply_publish_user_photo);
                                reply_object.put("publish_time", reply_time);
                                reply_object.put("spider_time", reply_spider_time);
                                reply_object.put("content", FileUtils.filterImg(reply_content));
                                reply_object.put("img_urls",img_urls);
                                reply_object.put("url", reply_url);
                                reply_object.put("parent_id", parent_id);
                                reply_object.put("ancestor_id", ancestor_id);

                                reply_nodes.add(reply_object);
                                // System.out.println(reply_time);
                            } catch (Exception e) {
                                continue;
                            }

                        }

                        JSONArray replys = new JSONArray();
                        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                        if (page >= 2) {
                            int threadNum = Integer.valueOf(PropertiesUtil.getVal("KaiDi.ReplyThreadCount"));
                            ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                            try {
                                List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                for (int i = 0; i < splitPages.size(); i++) {
                                    KaiDiDownLoader downLoader = this.cloneDownLoader();
                                    ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), content_url, id,speechesId);
                                    FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                    futureTasks.add(futureTask);
                                    executorService.submit(futureTask);
                                }

                                int count = 0;
                                for (FutureTask<JSONArray> futureTask : futureTasks) {
                                    JSONArray replyArrays = futureTask.get();
                                    for (int i = 0; i < replyArrays.size(); i++) {
                                        JSONObject object = replyArrays.getJSONObject(i);
                                        replys.add(object);
                                    }
                                }
                                if (replys.size() > 0 && replys != null) {
                                    for (int i = 0; i < replys.size(); i++) {
                                        JSONObject reply_object = replys.getJSONObject(i);
                                        reply_nodes.add(reply_object);
                                    }
                                }
                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                        JSONObject reply_object = reply_nodes.getJSONObject(i);
                                        Map<String, Object> map = reply_object;
                                        list.add(map);
                                    }
                                }
                                Collections.sort(list, new Comparator<Map<String, Object>>() {

                                    public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                        String s1 = o1.get("publish_time").toString();
                                        String s2 = o2.get("publish_time").toString();
                                        int flag = s1.compareTo(s2);
                                        return flag;
                                    }
                                });
                            }catch (Exception e){
                                logger.error(e.getMessage(),e);
                            }finally{
                                executorService.shutdown();
                            }


                        } else {
                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                for (int i = 0; i < reply_nodes.size(); i++) {
                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                    Map<String, Object> map = reply_object;
                                    list.add(map);
                                }
                            }
                        }
                        data.put("reply_nodes", list);


                    } catch (Exception e) {
                        logger.error(e.getMessage(), e);
                    }

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                } else {

                }


            }catch(Exception e){
                // logger.error(e.getMessage(),e);
                e.printStackTrace();
            }


            //}
        }catch(Exception e){
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(),e);
        }
        item.put("data", data);
        return item;
    }

    public KaiDiDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        KaiDiDownLoader loader =new KaiDiDownLoader();
        loader.cookies= (HashSet<Cookie>) this.cookies.clone();
        loader.headers= (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public static void main(String[] args) {
        KaiDiDownLoader downLoader = new KaiDiDownLoader();
        //List<String> urlList = downLoader.getUrl();
        //downLoader.generateTask(urlList);
        //http://club.kdnet.net/dispbbs.asp?s=share&id=9539541&boardid=1
        downLoader.crawlItem("http://club.kdnet.net/dispbbs.asp?s=share&id=12142834&boardid=1");
    }



    public class ReplyCrawler implements Callable<JSONArray>{
        private KaiDiDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;
        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();
            for(int k=0;k<pages.size();k++){
                //boolean flag = true;
                long future= System.currentTimeMillis()+Long.valueOf(PropertiesUtil.getVal("KaiDi.WaitTime"));
                //long remianing=Long.valueOf(PropertiesUtil.getVal("KaiDi.WaitTime"));
                //while(flag && remianing>=0) {
                try {
                    //remianing=future-System.currentTimeMillis();
                    String getUrl = url + "&page=" + pages.get(k);

                    Document document = null;
                    Page pageContent = null;

                    try {
                        pageContent = downLoader.getHtml(getUrl);

                        if (pageContent == null && pageContent.getStatusCode() != 200) {
                            pageContent = downLoader.getHtml(getUrl, true);
                        }
                    }catch (Exception e){
                        pageContent = downLoader.getHtml(getUrl, true);
                        e.printStackTrace();
                    }

                    //pageContent = downLoader.getHtml(getUrl, ProxyPool.proxy());


                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        document =  pageContent.getHtml().getDocument();
                        File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                        files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                        Elements replyInfos = document.select(".reply-box");
                        for (int i = 0; i < replyInfos.size(); i++) {
                            try {
                                JSONObject reply_object = new JSONObject();
                                String reply_id = replyInfos.get(i).attr("id");
                                String reply_publish_user_url = replyInfos.get(i).select(".posted-box").select(".posted-info").select(".name").select("span").get(0).select("a").get(0).attr("href");
                                String reply_publish_user_id = "";
                                String reply_publish_user_photo = "";
                                if (reply_publish_user_url != null && !"".equals(reply_publish_user_url)) {
                                    reply_publish_user_id = reply_publish_user_url.split("\\?")[1].split("=")[1];
                                    reply_publish_user_photo = "http://qc-cache.kdnet.net//upload/userface/"+reply_publish_user_id+".jpg?imageMogr2/crop/190x190";
                                        /*Document document1 = null;
                                        try {
                                            document1 = get(reply_publish_user_url);
                                            reply_publish_user_photo = document1.select("#content").select(".n_gb_c").select(".userinfo").select(".userpic").select("a").get(0).select("#userface_img_index").attr("src");
                                        }catch (Exception e){

                                        }*/
                                }
                                String reply_publish_user = replyInfos.get(i).select(".posted-box").select(".posted-info").select(".name").select("span").get(0).select("a").get(0).text();
                                String reply_time = replyInfos.get(i).select(".posted-box").select(".posted-info").text();
                                reply_time = reply_time.split("\\|")[3].split(" ")[1] + " " + reply_time.split("\\|")[3].split(" ")[2];
                                reply_time = reply_time.replaceAll("/","-");
                                String reply_spider_time = DateUtils.dateToString(new Date(), DateUtils.DATETIME_FORMAT);
                                String reply_content = replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").html();
                                JSONArray reply_img_urls=new JSONArray();
                                Elements replyImgElements=replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").select("img");
                                for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                    Element imgElement=replyImgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    reply_img_urls.add(img_url);
                                }
                                String reply_url = url + "#" + reply_id;

                                String parent_id1 = "";
                                String ancestor_id = "";
                                Elements reply_reply = replyInfos.get(i).select(".replycont-box").select(".replycont-box-r").select(".replycont-text").select(".quote-cont-box");
                                if (reply_reply.size() > 0 && reply_reply != null) {
                                    String parent_content_url = reply_reply.select(".quote-cont2").select(".quote-title").select("a").get(0).attr("href");
                                    parent_id1 = getUrlParams(parent_content_url).get("replyid").toString();
                                    ancestor_id = parent_id;
                                } else {
                                    parent_id1 = parent_id;
                                    ancestor_id = parent_id;
                                }
                                reply_object.put("id", reply_id);
                                reply_object.put("publish_user", reply_publish_user);
                                reply_object.put("publish_user_id", reply_publish_user_id);
                                reply_object.put("publish_user_photo", reply_publish_user_photo);
                                reply_object.put("publish_time", reply_time);
                                reply_object.put("spider_time", reply_spider_time);
                                reply_object.put("content", FileUtils.filterImg(reply_content));
                                reply_object.put("img_urls",reply_img_urls);
                                reply_object.put("url", reply_url);
                                reply_object.put("parent_id", parent_id1);
                                reply_object.put("ancestor_id", ancestor_id);
                                array.add(reply_object);
                                //flag = false;
                            } catch (Exception e) {
                                e.printStackTrace();
                                continue;
                            }
                        }
                    }else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                    } else {

                    }



                } catch (Exception e) {
                    //  logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    continue;
                }
                //}
            }
            return array;
        }

        public ReplyCrawler(KaiDiDownLoader downLoader, ArrayList<Integer> pages,String url,String parent_id,String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url=url;
            this.parent_id = parent_id;
            this.speeches_id=speeches_id;
        }
    }



}
